package douban;

import dao.JdbcDemo;
import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;
import model.Utils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.File;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;

/**
 * Created by liuyaowen on 2016/8/25.
 */
public class DoubanCrawler extends WebCrawler {
    private static final Pattern IMAGE_EXTENSIONS = Pattern.compile(".*\\.(bmp|gif|jpg|png)$");
    AtomicInteger index = new AtomicInteger(0);
    boolean is_debug = false;
    @Override
    public boolean shouldVisit(Page referringPage, WebURL url) throws Exception {
        String href = url.getURL().toLowerCase();
        if (IMAGE_EXTENSIONS.matcher(href).matches()) {
            return false;
        }

        return href.startsWith("https://www.douban.com/group/topic/") && href.endsWith("/");
    }

    @Override
    public void visit(Page page) {
        logger.info("正在解析的URL：{" + index + "}" + page.getWebURL().getURL());
        HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
        Document doc = Jsoup.parse(htmlParseData.getHtml());
        Element contents = doc.select("div[id=content]").first();

        DouBanContent douBanContent = new DouBanContent();
        //处理标题
        String title = handleTitle(contents);

        if (Utils.titleSet.contains(title)){
            logger.info("重复该{"+title+"},不再进行抓取...");
            return;
        }

        Utils.add_title_set(title);

        //处理内容
        String content = handleContent(contents, page.getWebURL().getURL());
        //处理评论
        Set<String> reviews = handleReview(doc.select("ul[id=comments]").select("li"));

        douBanContent.setTitle(title);
        douBanContent.setContent(content);
        douBanContent.setReviews(reviews);
        douBanContent.setCategory_id("2");
        index.addAndGet(1);
        try {
            //插入数据库
            if (!is_debug) {
                JdbcDemo.save_aws_question(douBanContent);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        logger.debug("=============");
    }

    private Set<String> handleReview(Elements elements) {
        Set<String> hashSet = new HashSet<>();
        if (elements.size() > 0 && !is_debug) {
            for (Element element : elements) {
                String review = element.select(".content p").text();
                hashSet.add(review);
                if (hashSet.size()>5){
                    break;
                }
            }
            logger.info("handleReview.size：{" + hashSet.size() + "}");
        }
        return hashSet;
    }

    private String handleContent(Element c, String sourceUrl) {
        StringBuffer sb = new StringBuffer();
        String content = c.select(".topic-doc p").toString();
        sb.append(content.replaceAll("<br>", "\n").replaceAll("<p>", "").replaceAll("</p>", ""));
        Elements elements = c.select(".topic-doc .topic-figure img");
        if (elements.size() > 0 && !is_debug) {
            int j = 0;
            sb.append("图片如下：\n");
            for (Element element : elements) {
                String url = element.attr("src");
                try {
                    String imgName = url.split("/")[url.split("/").length - 1];
                    String imgFileName = "c:\\xxoo\\" + imgName;
                    //下载到本地
                    logger.info("Downloader.download......(" + j + "){" + url + "}");
                    Downloader.download(url, imgName, "c:\\xxoo\\");
                    //上传到七牛
                    UploadUtils.upload(imgName, imgFileName);
                    //删除本地
                    File file = new File(imgFileName);
                    file.delete();
                    //写入文章内容
                    sb.append("[img]" + UploadUtils.QINIU_URL + imgName + "[/img]").append("\n");
                    j = ++j;
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }
        sb.append("\n来源：" + sourceUrl);
        return sb.toString();
    }

    private String handleTitle(Element c) {
        String title = c.select("h1").first().text();
        if (title.indexOf("...") > -1) {
            title = c.select(".tablecc").text().replaceAll("标题：", "");
        }
        logger.info("handleTitle：{" + title + "}");
        return title;
    }
}


